from MapReduce import Map, Reduce

import time
from multiprocessing import Process, Queue

if __name__ == '__main__':

    # 文档文件夹目录
    PATH = r"C:\Users\lenovo\Resource"
    # 确定Map进程数量
    for Map_num in range(1, 10):
        # 计时开始
        a = time.time()

        # 建立文档路径队列和词频统计结果队列
        in_q, res_q = Queue(), Queue()
        # 将路径写入文档路径队列
        for i in range(131610, 132610):
            in_q.put(rf"{PATH}\{i}.txt")
        # 向文档队列路径插入与Map进程数量相等的None，用于结束Map进程
        for i in range(Map_num):
            in_q.put(None)

        # 建立Map子进程，开始进程
        Mappers = []
        for i in range(Map_num):
            m = Process(target=Map, args=(in_q, res_q))
            Mappers.append(m)
        for m in Mappers:
            m.start()

        # Reduce子进程，收集所有词频统计结果，存入文档库后写进磁盘
        r = Process(target=Reduce, args=(res_q, Map_num))
        r.start()
        r.join()  # 主进程等待Reduce子进程结束，而无需等待Map进程结束

        # 计时结束，打印结果
        b = time.time()
        print(f"{Map_num} processes, time costs {b - a}s.")
